﻿/* ----------------------------------------------------------------------------
 * Copyright (c) Philip Stears 
 *
 * This source code is subject to terms and conditions of the Microsoft Public 
 * License (Ms-PL).
 * 
 * By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Microsoft Public License.
 * 
 * To request a copy of the Microsoft Public License, send a request by email
 * to license@connectbasic.net.
 *
 * This notice must not be removed from this software.
 * ---------------------------------------------------------------------------- */

namespace ConnectBasic.Execution
{
    using System;
    using System.Collections.Generic;
    using ConnectBasic.ObjectModel;
    using Microsoft.Scripting;

    /// <summary>
    /// Used inside the parser to support VBA's non-paranthesized functional call syntax.
    /// </summary>
    internal enum FunctionCallBehavior
    {
        ParanthesisRequired,
        ParanthesisOptional
    }

    /// <summary>
    /// Provides a parser which parses ConnectBasic code.
    /// </summary>
    public class BasicParser
    {
        private const BasicTokenKind DEFAULT_MODULE_ACCESS_LEVEL = BasicTokenKind.KeywordPublic;

        #region Static Implementation

        /// <summary>
        /// Parse the source code in the given source unit as a standard module.
        /// </summary>
        /// <param name="source">The source code to parse.</param>
        /// <param name="errorSink">An object which will receive information about any errors that occur during parsing.</param>
        /// <returns></returns>
        public static StandardModuleCodeUnit ParseStandardModule(SourceUnit source, ErrorSink errorSink)
        {
            return ParseStandardModule(source, errorSink, null);
        }

        /// <summary>
        /// Parse the source code in the given source unit as a standard module.
        /// </summary>
        /// <param name="source">The source code to parse.</param>
        /// <param name="errorSink">An object which will receive information about any errors that occur during parsing.</param>
        /// <param name="tokenSink">An object which will receive information about the tokens retrieved during parsing.</param>
        /// <returns></returns>
        public static StandardModuleCodeUnit ParseStandardModule(SourceUnit source, ErrorSink errorSink, ITokenSink tokenSink)
        {
            if (source == null) throw new ArgumentNullException("source");

            // Parse the module using an instance of the parser
            return (new BasicParser(source, errorSink ?? ErrorSink.Default, tokenSink)).ParseStandardModuleCore();
        }

        #endregion

        #region Instance Implementation

        private SourceUnit mSource;
        private ErrorSink mSink;
        private BasicTokenizer mTokenizer;
        private ITokenSink mTokenSink;
        private ExpressionBase mCurrentExpression;
        private bool mOptIsByRefDefault = true;
        private int mOptStringCompare = 1; // Option Compare Text
        private bool mOptExplicit;
        private bool mOptStrict;

        private BasicParser(SourceUnit source, ErrorSink sink, ITokenSink tokenSink)
        {
            if (source == null) throw new ArgumentNullException("source");

            mSource = source;
            mSink = sink;
            mTokenizer = new BasicTokenizer(source, sink);
            mTokenSink = tokenSink;
        }

        private StandardModuleCodeUnit ParseStandardModuleCore()
        {
            var tokenizer = mTokenizer;
            var constants = new List<ConstantStatement>();
            var methods = new List<MethodDefinitionNode>();
            var variables = new List<VariableDeclarationStatement>();
            var allowOption = true;

            // Make sure we've read the first token as all of our code
            // works on a token already loaded into the tokenizer
            this.NextToken();

            // Read top-level statements
            for (; ; )
            {
                BasicTokenKind? currentMemberAccessLevel = null;

                // Eat whitespace, new-lines, and comments
                MaybeEatWhitespaceNewLinesOrComment();

                // See if we've got an access modifier
                if (tokenizer.CurrentToken.Kind == BasicTokenKind.KeywordPublic || tokenizer.CurrentToken.Kind == BasicTokenKind.KeywordPrivate)
                {
                    currentMemberAccessLevel = tokenizer.CurrentToken.Kind;
                    this.NextToken();
                    this.EatWhitespace();

                    if (tokenizer.CurrentToken.Kind == BasicTokenKind.Identifier)
                    {
                        variables.Add(this.ParseVariable());
                        continue;
                    }
                    else if (tokenizer.CurrentToken.Kind == BasicTokenKind.KeywordDim)
                    {
                        this.NextToken();
                        this.EatWhitespace();
                        variables.Add(this.ParseVariable());
                        continue;
                    }
                }

                // See what we're dealing wth
                switch (tokenizer.CurrentToken.Kind)
                {
                    case BasicTokenKind.Comment:

                        // Can't follow an access modifier
                        if (currentMemberAccessLevel.HasValue)
                        {
                            this.OnUnexpectedToken(tokenizer.CurrentToken, false);
                        }

                        this.NextToken();
                        this.EatNewLines();
                        break;

                    case BasicTokenKind.KeywordDim:
                        this.NextToken();
                        this.EatWhitespace();
                        variables.Add(this.ParseVariable());
                        break;

                    case BasicTokenKind.KeywordConst:
                        this.NextToken();
                        this.EatWhitespace();
                        constants.Add(this.ParseConst());
                        break;

                    case BasicTokenKind.KeywordOption:
                        if (!allowOption)
                        {
                            this.OnUnexpectedToken(tokenizer.CurrentToken, false);
                        }

                        this.NextToken();
                        this.EatWhitespace();
                        this.ParseOption();
                        break;

                    case BasicTokenKind.KeywordFunction:
                        this.NextToken();
                        this.EatWhitespace();
                        methods.Add(this.ParseFunction(currentMemberAccessLevel.HasValue ? currentMemberAccessLevel.Value : DEFAULT_MODULE_ACCESS_LEVEL));
                        break;

                    case BasicTokenKind.KeywordSub:
                        this.NextToken();
                        this.EatWhitespace();
                        methods.Add(this.ParseSub(currentMemberAccessLevel.HasValue ? currentMemberAccessLevel.Value : DEFAULT_MODULE_ACCESS_LEVEL));
                        break;

                    case BasicTokenKind.EndOfFile:

                        // Can't follow an access modifier
                        if (currentMemberAccessLevel.HasValue)
                        {
                            this.OnUnexpectedToken(tokenizer.CurrentToken, false);
                        }

                        goto ExitFor;

                    default:
                        this.OnUnexpectedToken(tokenizer.CurrentToken, true);
                        break;
                }
            }
        ExitFor:

            // Parsed!
            return new StandardModuleCodeUnit(mSource.Path, methods, constants, variables);
        }

        private ConstantStatement ParseConst()
        {

            // Get the constant name
            this.ExpectToken(BasicTokenKind.Identifier);
            var identifierToken = mTokenizer.CurrentToken;

            // Get the constant type
            this.NextToken();
            this.EatWhitespace();
            this.SkipToken(BasicTokenKind.KeywordAs);
            this.EatWhitespace();
            this.ExpectToken(BasicTokenKind.Identifier);
            var type = this.ParseType();

            // Get the constant value expression
            this.MaybeEatWhitespace();
            this.SkipToken(BasicTokenKind.AssignOrEqual);
            this.MaybeEatWhitespace();
            var constantExpression = this.ParseExpression();

            // Finish consuming the line
            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            // Construct and return the constant
            return new ConstantStatement(identifierToken.ValueString, type, constantExpression);
        }

        private void ParseOption()
        {
            switch (mTokenizer.CurrentToken.Kind)
            {
                case BasicTokenKind.KeywordCompare:
                    this.NextToken();
                    this.EatWhitespace();

                    if (mTokenizer.CurrentToken.Kind == BasicTokenKind.KeywordBinary)
                    {
                        mOptStringCompare = 0;
                        this.MaybeEatWhitespaceOrComment();
                        this.EatNewLines();
                    }
                    else if (mTokenizer.CurrentToken.Kind == BasicTokenKind.KeywordText)
                    {
                        mOptStringCompare = 1;
                        this.MaybeEatWhitespaceOrComment();
                        this.EatNewLines();
                    }
                    else
                    {
                        this.OnUnexpectedToken(mTokenizer.CurrentToken, true);
                        this.MaybeEatWhitespaceOrComment();
                        this.EatNewLines();
                    }
                    return;

                case BasicTokenKind.KeywordExplicit:
                    mOptExplicit = true;
                    this.NextToken();
                    this.MaybeEatWhitespaceOrComment();
                    this.EatNewLines();
                    return;

                case BasicTokenKind.KeywordStrict:
                    mOptStrict = true;
                    this.NextToken();
                    this.MaybeEatWhitespaceOrComment();
                    this.EatNewLines();
                    return;

                default:
                    this.OnUnexpectedToken(mTokenizer.CurrentToken, true);
                    this.MaybeEatWhitespaceOrComment();
                    this.EatNewLines();
                    return;
            }
        }

        private VariableDeclarationStatement ParseVariable()
        {
            this.ExpectToken(BasicTokenKind.Identifier);
            var identifierToken = mTokenizer.CurrentToken;
            this.NextToken();

            var type = UnboundType.Variant;

            if (this.MaybeEatWhitespace() && this.MaybeSkipToken(BasicTokenKind.KeywordAs))
            {
                this.EatWhitespace();
                type = this.ParseType();
                this.MaybeEatWhitespace();
            }

            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            return new VariableDeclarationStatement(identifierToken.ValueString, type);
        }

        private MethodDefinitionNode ParseFunction(BasicTokenKind accessLevel)
        {
            UnboundType returnType = UnboundType.Variant;
            List<ParameterDefinition> parameters = new List<ParameterDefinition>();
            IEnumerable<StatementNodeBase> statements;

            // Make sure we've got an identifier
            this.ExpectToken(BasicTokenKind.Identifier);
            var identifierToken = mTokenizer.CurrentToken;

            // Parse the parameter list
            this.NextToken();
            this.MaybeEatWhitespace();
            this.SkipToken(BasicTokenKind.ParanthesisOpen);
            parameters.AddRange(this.ParseParameterList());
            parameters.AddRange(this.ParseOptionalParameterList());
            this.SkipToken(BasicTokenKind.ParanthesisClose);

            // Parse the return type
            this.EatWhitespace();

            if (this.MaybeSkipToken(BasicTokenKind.KeywordAs))
            {
                this.EatWhitespace();
                this.ExpectToken(BasicTokenKind.Identifier);
                returnType = this.ParseType();
            }

            // Get rid of trailing whitespace and comments
            this.MaybeEatWhitespaceOrComment();

            // Expect a new line
            this.Eat(BasicTokenKind.NewLines);

            // Statements
            statements = this.ParseStatements();

            // Make sure the end of the function is reached
            this.Eat(BasicTokenKind.KeywordEnd);
            this.EatWhitespace();
            this.Eat(BasicTokenKind.KeywordFunction);

            // Return the function
            return new MethodDefinitionNode(
                identifierToken.ValueString,
                returnType,
                parameters,
                statements);
        }

        private MethodDefinitionNode ParseSub(BasicTokenKind accessLevel)
        {
            IEnumerable<ParameterDefinition> parameters;
            IEnumerable<StatementNodeBase> statements;

            // Make sure we've got an identifier
            this.ExpectToken(BasicTokenKind.Identifier);
            var identifierToken = mTokenizer.CurrentToken;

            // Parse the parameter list
            this.NextToken();
            this.MaybeEatWhitespace();
            this.SkipToken(BasicTokenKind.ParanthesisOpen);
            parameters = this.ParseParameterList();
            this.SkipToken(BasicTokenKind.ParanthesisClose);

            // Get rid of trailing whitespace and comments
            this.MaybeEatWhitespaceOrComment();

            // Expect a new line
            this.Eat(BasicTokenKind.NewLines);

            // Statements
            statements = this.ParseStatements();

            // Make sure the end of the function is reached
            this.Eat(BasicTokenKind.KeywordEnd);
            this.EatWhitespace();
            this.Eat(BasicTokenKind.KeywordSub);

            // Return the function
            return new MethodDefinitionNode(
                identifierToken.ValueString,
                UnboundType.Void,
                parameters,
                statements);
        }

        private IEnumerable<StatementNodeBase> ParseStatements()
        {
            var statements = new List<StatementNodeBase>();

            // Read statements
            this.MaybeEatWhitespaceNewLinesOrComment();

            for (; ; )
            {
                this.MaybeEatWhitespaceOrComment();

                switch (mTokenizer.CurrentToken.Kind)
                {
                    case BasicTokenKind.KeywordFor:
                        statements.Add(this.ParseFor());
                        break;

                    case BasicTokenKind.KeywordOn:
                        this.NextToken();
                        this.EatWhitespace();
                        this.ExpectToken(BasicTokenKind.KeywordError);
                        this.NextToken();
                        this.EatWhitespace();

                        if (this.MaybeSkipToken(BasicTokenKind.KeywordResume))
                        {
                            this.EatWhitespace();
                            this.SkipToken(BasicTokenKind.KeywordNext);

                            statements.Add(new OnErrorResumeNextStatement());

                            this.NextToken();
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else if (this.MaybeSkipToken(BasicTokenKind.KeywordGoTo))
                        {
                            this.EatWhitespace();
                            this.ExpectToken(BasicTokenKind.Identifier);

                            statements.Add(new OnErrorGoToStatement(mTokenizer.CurrentToken.ValueString));

                            this.NextToken();
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else
                        {
                            this.OnUnexpectedToken(mTokenizer.CurrentToken, true);
                        }

                        break;

                    case BasicTokenKind.KeywordExit:
                        this.NextToken();
                        this.EatWhitespace();

                        if (this.MaybeSkipToken(BasicTokenKind.KeywordSub))
                        {
                            statements.Add(new ExitScopeStatement(BasicTokenKind.KeywordSub));
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else if (this.MaybeSkipToken(BasicTokenKind.KeywordFunction))
                        {
                            statements.Add(new ExitScopeStatement(BasicTokenKind.KeywordFunction));
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else if (this.MaybeSkipToken(BasicTokenKind.KeywordFor))
                        {
                            statements.Add(new ExitScopeStatement(BasicTokenKind.KeywordFor));
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else if (this.MaybeSkipToken(BasicTokenKind.KeywordWhile))
                        {
                            statements.Add(new ExitScopeStatement(BasicTokenKind.KeywordWhile));
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }
                        else
                        {
                            this.OnUnexpectedToken(mTokenizer.CurrentToken, true);
                        }
                        break;

                    case BasicTokenKind.KeywordCall:
                        {
                            var callKeyword = mTokenizer.CurrentToken;

                            // Get the identifier
                            this.NextToken();
                            this.MaybeEatWhitespace();
                            this.ExpectToken(BasicTokenKind.Identifier);
                            var identifierToken = mTokenizer.CurrentToken;

                            this.NextToken();
                            this.MaybeEatWhitespace();

                            if (this.MaybeSkipToken(BasicTokenKind.ParanthesisOpen))
                            {

                                // With arguments
                                var args = this.ParseMethodArgumentsWithParan();

                                // The token currently in the tokenizer is the closing paranthesis
                                // (assuming the code is correct anyway)
                                var callExpressionSpan = SourceSpan.Invalid;
                                var callStatementSpan = SourceSpan.Invalid;

                                if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
                                {
                                    callExpressionSpan = new SourceSpan(identifierToken.Span.Start, mTokenizer.CurrentToken.Span.End);
                                    callStatementSpan = new SourceSpan(callKeyword.Span.Start, callExpressionSpan.End);

                                    this.NextToken();
                                }

                                // Add the call statement
                                var callExpression = new CallExpression(callExpressionSpan, identifierToken.ValueString, args);
                                var callStatement = new CallStatement(callStatementSpan, callExpression);
                                statements.Add(callStatement);
                            }
                            else if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Dot)
                            {

                                // Invoke
                                this.NextToken();
                                statements.Add(new InvokeStatement(this.ParseInvoke(identifierToken, FunctionCallBehavior.ParanthesisRequired)));
                            }
                            else
                            {

                                // No arguments
                                var callExpressionSpan = identifierToken.Span;
                                var callStatementSpan = new SourceSpan(callKeyword.Span.Start, callExpressionSpan.End);
                                var callExpression = new CallExpression(callExpressionSpan, identifierToken.ValueString, EmptyArray<ExpressionBase>.Instance);
                                var callStatement = new CallStatement(callStatementSpan, callExpression);

                                statements.Add(callStatement);
                            }

                            // The call must be the last meaningful thing on the line
                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }

                        break;

                    case BasicTokenKind.KeywordSet:
                        {
                            var setKeyword = mTokenizer.CurrentToken;

                            // Get the identifier
                            this.NextToken();
                            this.MaybeEatWhitespace();
                            this.ExpectToken(BasicTokenKind.Identifier);
                            BasicToken identifierToken = mTokenizer.CurrentToken;

                            // Get the '='
                            this.NextToken();
                            this.MaybeEatWhitespace();
                            this.ExpectToken(BasicTokenKind.AssignOrEqual);

                            // Read the assignment statement
                            this.NextToken();
                            this.MaybeEatWhitespace();
                            var assignExpression = this.ParseExpression();

                            // Setup debugging information
                            var span = setKeyword.Span;

                            if (assignExpression != null)
                            {
                                span = new SourceSpan(span.Start, assignExpression.Span.End);
                            }                            

                            statements.Add(new SetStatement(
                                span,
                                identifierToken.ValueString,
                                assignExpression
                                ));

                            this.MaybeEatWhitespaceOrComment();
                            this.EatNewLines();
                        }

                        break;

                    case BasicTokenKind.Identifier:
                        {
                            BasicToken identifierToken = mTokenizer.CurrentToken;

                            this.NextToken();

                            // Label?
                            if (this.MaybeSkipToken(BasicTokenKind.Colon))
                            {
                                statements.Add(new LabelStatement(identifierToken.ValueString));

                                this.MaybeEatWhitespaceOrComment();
                                this.EatNewLines();
                            }
                            else
                            {
                                this.MaybeEatWhitespace();

                                // Function call?
                                if (this.MaybeSkipToken(BasicTokenKind.ParanthesisOpen))
                                {

                                    // With arguments
                                    var args = this.ParseMethodArgumentsWithParan();

                                    // The token currently in the tokenizer is the closing paranthesis
                                    // (assuming the code is correct anyway)
                                    var callExpressionSpan = SourceSpan.Invalid;

                                    if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
                                    {
                                        callExpressionSpan = new SourceSpan(identifierToken.Span.Start, mTokenizer.CurrentToken.Span.End);
                                        this.NextToken();
                                    }

                                    // Add the call statement
                                    var callExpression = new CallExpression(callExpressionSpan, identifierToken.ValueString, args);
                                    var callStatement = new CallStatement(callExpressionSpan, callExpression);
                                    statements.Add(callStatement);
                                }
                                // Assignment?
                                else if (mTokenizer.CurrentToken.Kind == BasicTokenKind.AssignOrEqual)
                                {
                                    this.NextToken();
                                    this.MaybeEatWhitespace();

                                    // Parse the RHS
                                    var assignExpression = this.ParseExpression();

                                    // Setup debugging information
                                    var span = identifierToken.Span;

                                    if (assignExpression != null && assignExpression.Span.IsValid)
                                    {
                                        span = new SourceSpan(span.Start, assignExpression.Span.End);
                                    }   

                                    statements.Add(new AssignStatement(
                                        span,
                                        identifierToken.ValueString,
                                        assignExpression
                                        ));
                                }
                                // Member invoke
                                else if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Dot)
                                {
                                    this.NextToken();

                                    statements.Add(new InvokeStatement(
                                        this.ParseInvoke(identifierToken, FunctionCallBehavior.ParanthesisOptional)
                                        ));
                                }
                                // Function call - no parentheses?
                                else
                                {

                                    // With arguments
                                    var args = this.ParseMethodArgumentsNoParan();

                                    // The last token in the args is what we should use as the ending
                                    // location
                                    var callExpressionSpan = args.Count > 0 ?
                                        new SourceSpan(identifierToken.Span.Start, args[args.Count - 1].Span.End) :
                                        new SourceSpan(identifierToken.Span.Start, identifierToken.Span.End);

                                    // Add the call statement
                                    var callExpression = new CallExpression(callExpressionSpan, identifierToken.ValueString, args);
                                    var callStatement = new CallStatement(callExpressionSpan, callExpression);
                                    statements.Add(callStatement);
                                }

                                // Done for this line
                                this.MaybeEatWhitespaceOrComment();
                                this.EatNewLines();
                            }
                        }

                        break;

                    case BasicTokenKind.KeywordDim:

                        this.NextToken();
                        this.EatWhitespace();
                        statements.Add(this.ParseVariable());
                        break;

                    case BasicTokenKind.Comment:
                        this.NextToken();
                        this.EatNewLines();
                        break;

                    case BasicTokenKind.KeywordIf:
                        statements.Add(this.ParseIf());
                        break;

                    case BasicTokenKind.KeywordWhile:
                        statements.Add(this.ParseWhile());
                        break;

                    case BasicTokenKind.KeywordDo:
                        statements.Add(this.ParseDo());
                        break;

                    case BasicTokenKind.NewLines:
                        this.EatNewLines();
                        break;

                    default:

                        // If we don't recognize it as a statement, let the calling
                        // production deal with it
                        return statements;
                }
            }
        }

        private IEnumerable<ParameterDefinition> ParseParameterList()
        {
            var parameters = new List<ParameterDefinition>();

            // Read parameters
            for (; ; )
            {
                bool isByRef = mOptIsByRefDefault;

                switch (mTokenizer.CurrentToken.Kind)
                {
                    case BasicTokenKind.KeywordByRef:
                        this.NextToken();
                        this.EatWhitespace();
                        this.ExpectToken(BasicTokenKind.Identifier);
                        isByRef = true;

                        goto case BasicTokenKind.Identifier;

                    case BasicTokenKind.KeywordByVal:
                        this.NextToken();
                        this.EatWhitespace();
                        this.ExpectToken(BasicTokenKind.Identifier);
                        isByRef = false;

                        goto case BasicTokenKind.Identifier;

                    case BasicTokenKind.Identifier:
                        var tokenIdentifier = mTokenizer.CurrentToken;
                        var parameterType = UnboundType.Variant;

                        this.NextToken();

                        if (this.MaybeEatWhitespace() && this.MaybeSkipToken(BasicTokenKind.KeywordAs))
                        {
                            this.EatWhitespace();
                            parameterType = this.ParseType();
                            this.MaybeEatWhitespace();
                        }

                        parameters.Add(new ParameterDefinition(tokenIdentifier.ValueString, parameterType, isByRef, false));

                        if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Comma)
                        {

                            // Got another parameter, read the next token
                            this.NextTokenAndMaybeEatWhitespace();
                            break;
                        }
                        else
                        {

                            // If we've hit a token we don't reconize, assume
                            // we've hit the end of the parameter list, and let the calling
                            // production deal with it.
                            return parameters;
                        }

                    default:

                        // If we've hit a token we don't reconize, assume
                        // we've hit the end of the parameter list, and let the calling
                        // production deal with it.
                        return parameters;
                }
            }
        }

        private IEnumerable<ParameterDefinition> ParseOptionalParameterList()
        {
            var parameters = new List<ParameterDefinition>();

            // Read parameters
            for (; ; )
            {
                bool isByRef = mOptIsByRefDefault;

                if (!this.MaybeSkipToken(BasicTokenKind.KeywordOptional))
                {
                    return parameters.ToArray();
                }

                this.EatWhitespace();

                switch (mTokenizer.CurrentToken.Kind)
                {
                    case BasicTokenKind.KeywordByRef:
                        this.NextToken();
                        this.EatWhitespace();
                        this.ExpectToken(BasicTokenKind.Identifier);
                        isByRef = true;

                        goto case BasicTokenKind.Identifier;

                    case BasicTokenKind.KeywordByVal:
                        this.NextToken();
                        this.EatWhitespace();
                        this.ExpectToken(BasicTokenKind.Identifier);
                        isByRef = false;

                        goto case BasicTokenKind.Identifier;

                    case BasicTokenKind.Identifier:
                        var tokenIdentifier = mTokenizer.CurrentToken;
                        var parameterType = UnboundType.Variant;

                        this.NextToken();

                        if (this.MaybeEatWhitespace() && this.MaybeSkipToken(BasicTokenKind.KeywordAs))
                        {
                            this.EatWhitespace();
                            parameterType = this.ParseType();
                            this.MaybeEatWhitespace();
                        }

                        this.SkipToken(BasicTokenKind.AssignOrEqual);
                        this.EatWhitespace();

                        var defaultValue = this.ParseExpression();

                        parameters.Add(new OptionalParameterDefinition(tokenIdentifier.ValueString, defaultValue, parameterType, isByRef, false));

                        if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Comma)
                        {

                            // Got another parameter, read the next token
                            this.NextTokenAndMaybeEatWhitespace();
                            break;
                        }
                        else
                        {

                            // If we've hit a token we don't reconize, assume
                            // we've hit the end of the parameter list, and let the calling
                            // production deal with it.
                            return parameters;
                        }

                    default:

                        // If we've hit a token we don't reconize, assume
                        // we've hit the end of the parameter list, and let the calling
                        // production deal with it.
                        return parameters;
                }
            }
        }

        private ForStatement ParseFor()
        {
            this.NextToken();
            this.EatWhitespace();

            // The next token should be the identifier which tells us which
            // variable to use as a loop counter
            this.ExpectToken(BasicTokenKind.Identifier);
            var loopIdentifier = mTokenizer.CurrentToken;
            this.NextToken();
            this.MaybeEatWhitespace();

            // Now we should get the starting point
            this.SkipToken(BasicTokenKind.AssignOrEqual);
            this.MaybeEatWhitespace();

            var startExpression = this.ParseExpression(); // NOTE: Automatically eats trailing whitespace

            // Now we should get the 'To' keyword
            this.SkipToken(BasicTokenKind.KeywordTo);
            this.EatWhitespace();

            // And the terminating condition
            var endExpression = this.ParseExpression(); // NOTE: Automatically eats trailing whitespace

            // And, optionally, a Step condition
            var stepExpression = (ExpressionBase)null;

            if (this.MaybeSkipToken(BasicTokenKind.KeywordStep))
            {
                this.EatWhitespace();
                stepExpression = this.ParseExpression(); // NOTE: Automatically eats trailing whitespace
            }
            else
            {
                this.EatNewLines();
            }

            // Parse statements
            var statements = this.ParseStatements();

            // Validate the lead out
            this.SkipToken(BasicTokenKind.KeywordNext);

            if (this.MaybeEatWhitespace())
            {
                if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Identifier)
                {

                    // If they used the "Next x" syntaxt, make sure the identifier they provided matches
                    // the identifier being used as the loop counter
                    if (!string.Equals(mTokenizer.CurrentToken.ValueString, loopIdentifier.ValueString, StringComparison.OrdinalIgnoreCase))
                    {
                        this.OnUnexpectedIdentifier(loopIdentifier.ValueString, mTokenizer.CurrentToken, true);
                    }
                    else
                    {
                        this.NextToken();
                        this.MaybeEatWhitespace();
                    }
                }
            }

            // Return the for-statement
            return new ForStatement(loopIdentifier.ValueString, startExpression, endExpression, stepExpression, new Block(statements));
        }

        private StatementNodeBase ParseDo()
        {
            ExpressionBase condition;
            Block statements;

            // Consume the lead-in
            this.ExpectToken(BasicTokenKind.KeywordDo);
            this.NextToken();
            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            // Statements
            statements = new Block(this.ParseStatements());

            // Validate the lead-out
            this.SkipToken(BasicTokenKind.KeywordLoop);
            this.EatWhitespace();

            if (this.MaybeSkipToken(BasicTokenKind.KeywordUntil))
            {
                this.EatWhitespace();

                // Condition
                condition = this.ParseExpression();
                this.MaybeEatWhitespaceOrComment();
                this.EatNewLines();

                // Return the do...until-statement
                return new DoUntilStatement(condition, statements);
            }
            else if (this.MaybeSkipToken(BasicTokenKind.KeywordWhile))
            {
                this.EatWhitespace();

                // Condition
                condition = this.ParseExpression();
                this.MaybeEatWhitespaceOrComment();
                this.EatNewLines();

                // Return the do...while-statement
                return new DoWhileStatement(condition, statements);
            }
            else
            {
                this.OnUnexpectedToken(mTokenizer.CurrentToken, true);
                this.MaybeEatWhitespaceOrComment();
                this.EatNewLines();
                return null;
            }
        }

        private WhileStatement ParseWhile()
        {
            ExpressionBase condition;
            Block statements;

            // Consume the lead-in
            this.ExpectToken(BasicTokenKind.KeywordWhile);
            this.NextToken();
            this.EatWhitespace();

            // Condition
            condition = this.ParseExpression();

            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            // Statements
            statements = new Block(this.ParseStatements());

            // Validate the lead-out
            this.SkipToken(BasicTokenKind.KeywordWend);
            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            // Return the while-statement
            return new WhileStatement(condition, statements);
        }

        private IfStatement ParseIf()
        {
            List<ConditionalBlock> conditionalBlocks = new List<ConditionalBlock>();
            Block elseBlock = null;
            ExpressionBase condition;

            SourceLocation openingStart;
            SourceLocation openingEnd;

            // Consume the lead-in
            this.ExpectToken(BasicTokenKind.KeywordIf);
            openingStart = mTokenizer.CurrentToken.Span.Start;

            this.NextToken();
            this.EatWhitespace();

            // Main block (1)
            condition = this.ParseExpression();

            this.ExpectToken(BasicTokenKind.KeywordThen);
            openingEnd = mTokenizer.CurrentToken.Span.End;
            this.NextToken();

            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            conditionalBlocks.Add(new ConditionalBlock(condition, this.ParseStatements()));

            // ElseIf (0...n)
            for (; ; )
            {
                if (this.MaybeSkipToken(BasicTokenKind.KeywordElseIf))
                {
                    this.EatWhitespace();

                    condition = this.ParseExpression();

                    this.SkipToken(BasicTokenKind.KeywordThen);

                    this.MaybeEatWhitespaceOrComment();
                    this.EatNewLines();

                    conditionalBlocks.Add(new ConditionalBlock(condition, this.ParseStatements()));
                }
                else
                {
                    break;
                }
            }

            // Else (0 or 1)
            if (this.MaybeSkipToken(BasicTokenKind.KeywordElse))
            {
                this.MaybeEatWhitespaceOrComment();
                this.EatNewLines();

                elseBlock = new Block(this.ParseStatements());
            }

            // Validate the lead-out
            this.SkipToken(BasicTokenKind.KeywordEnd);
            this.SkipToken(BasicTokenKind.Whitespace);
            this.SkipToken(BasicTokenKind.KeywordIf);
            this.MaybeEatWhitespaceOrComment();
            this.EatNewLines();

            // Return the if-statement
            return new IfStatement(new SourceSpan(openingStart, openingEnd), conditionalBlocks, elseBlock);
        }

        /// <summary>
        /// Parses the arguments in a unparanthesized method call.
        /// </summary>
        /// <returns></returns>
        private IList<ExpressionBase> ParseMethodArgumentsNoParan()
        {
            List<ExpressionBase> arguments = new List<ExpressionBase>();

            // Might have some leading whitespace
            this.MaybeEatWhitespace();

            // Empty argument list?
            var firstArg = this.ParseExpression();

            if (firstArg == null)
            {
                this.MaybeSkipToken(BasicTokenKind.Comment);
                this.ExpectToken(BasicTokenKind.NewLines);
                return EmptyArray<ExpressionBase>.Instance;
            }
            else
            {
                arguments.Add(firstArg);

                if (mTokenizer.CurrentToken.Kind == BasicTokenKind.Comma)
                {
                    this.NextToken();
                    this.MaybeEatWhitespace();
                }
                else
                {
                    this.MaybeSkipToken(BasicTokenKind.Comment);
                    this.ExpectToken(BasicTokenKind.NewLines);
                    return arguments.ToArray();
                }
            }

            // Parse remaining arguments
            while (this.mTokenizer.CurrentToken.Kind != BasicTokenKind.EndOfFile)
            {
                arguments.Add(this.ParseExpression());
                this.MaybeEatWhitespace();

                if (this.MaybeSkipToken(BasicTokenKind.Comment))
                {
                    this.ExpectToken(BasicTokenKind.NewLines);
                    return arguments.ToArray();
                }
                else if (mTokenizer.CurrentToken.Kind == BasicTokenKind.NewLines)
                {
                    return arguments.ToArray();
                }
                else
                {
                    this.SkipToken(BasicTokenKind.Comma);
                    this.MaybeEatWhitespace();
                }
            }

            // If we got here, we hit the end of the stream unexpectedly, return as much as we got
            return arguments.ToArray();
        }

        /// <summary>
        /// Parses the arguments in a paranthesized method call and leaves the 
        /// tokenizer positioned on the closing paranthesis. 
        /// </summary>
        /// <returns>
        /// <para>
        /// The opening paranthesis should have already been consumed.
        /// </para>
        /// </returns>
        private IList<ExpressionBase> ParseMethodArgumentsWithParan()
        {
            List<ExpressionBase> arguments = new List<ExpressionBase>();

            // Might have some leading whitespace.
            this.MaybeEatWhitespace();

            // Empty argument list?
            if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
            {
                return EmptyArray<ExpressionBase>.Instance;
            }

            // Parse arguments until we hit a closing paranthesis
            while (this.mTokenizer.CurrentToken.Kind != BasicTokenKind.EndOfFile)
            {
                arguments.Add(this.ParseExpression());
                this.MaybeEatWhitespace();

                if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
                {
                    return arguments.ToArray();
                }
                else
                {
                    this.SkipToken(BasicTokenKind.Comma);
                    this.MaybeEatWhitespace();
                }
            }

            // If we got here, we hit the end of the stream unexpectedly, return as much as we got
            return arguments.ToArray();
        }

        private UnboundType ParseType()
        {
            this.ExpectToken(BasicTokenKind.Identifier);

            var identifierToken = mTokenizer.CurrentToken;

            this.NextToken();

            // TODO: Array types!
            if (this.MaybeSkipToken(BasicTokenKind.ParanthesisOpen))
            {
                this.SkipToken(BasicTokenKind.ParanthesisClose);

                return new UnboundType(identifierToken.ValueString, 1);
            }
            else
            {
                return new UnboundType(identifierToken.ValueString);
            }
        }

        #region Expression Parsing

        private ExpressionBase ParseExpression()
        {
            this.ParseExpression_Comparators();

            // Return the expression that was constructed
            var result = mCurrentExpression;
            mCurrentExpression = null;
            return result;
        }

        private void ParseExpression_Comparators()
        {
            this.ParseExpression_Concat();
            this.ParseExpression_Comparators_Prime();
        }

        private void ParseExpression_Comparators_Prime()
        {
            switch (mTokenizer.CurrentToken.Kind)
            {
                case BasicTokenKind.LessThan:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.LessThan, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;

                case BasicTokenKind.LessThanOrEqualTo:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.LessThanOrEqualTo, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;

                case BasicTokenKind.GreaterThan:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.GreaterThan, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;

                case BasicTokenKind.GreaterThanOrEqualTo:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.GreaterThanOrEqualTo, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;

                case BasicTokenKind.AssignOrEqual:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.EqualTo,
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;

                case BasicTokenKind.NotEqualTo:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Concat();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.NotEqualTo,
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Comparators_Prime();

                    } break;
            }
        }

        private void ParseExpression_Concat()
        {
            this.ParseExpression_Add();
            this.ParseExpression_Concat_Prime();
        }

        private void ParseExpression_Concat_Prime()
        {
            if (this.MaybeSkipToken(BasicTokenKind.Concatenate))
            {
                // Get and reset the LHS
                var left = mCurrentExpression;
                mCurrentExpression = null;

                // Parse the RHS
                this.EatWhitespace();
                this.ParseExpression_Add();

                // Setup the expression
                mCurrentExpression = new BinaryOperation(
                    MakeBinaryExpressionSpan(left, mCurrentExpression), 
                    BinaryOperationKind.Concatenate, 
                    left, 
                    mCurrentExpression);

                // Recurse
                this.ParseExpression_Concat_Prime();
            }
        }

        private void ParseExpression_Add()
        {
            this.ParseExpression_Power();
            this.ParseExpression_Add_Prime();
        }

        private void ParseExpression_Add_Prime()
        {
            switch (mTokenizer.CurrentToken.Kind)
            {
                case BasicTokenKind.Add:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Power();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.Add, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Add_Prime();

                    } break;

                case BasicTokenKind.Subtract:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        this.ParseExpression_Power();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.Subtract, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Add_Prime();

                    } break;
            }
        }

        private void ParseExpression_Power()
        {
            this.ParseExpression_Mult();
            this.ParseExpression_Power_Prime();
        }

        private void ParseExpression_Power_Prime()
        {
            if (this.MaybeSkipToken(BasicTokenKind.Power))
            {
                // Get and reset the LHS
                var left = mCurrentExpression;
                mCurrentExpression = null;

                // Parse the RHS
                this.EatWhitespace();
                this.ParseExpression_Mult();

                // Setup the expression
                mCurrentExpression = new BinaryOperation(
                    MakeBinaryExpressionSpan(left, mCurrentExpression), 
                    BinaryOperationKind.Power, 
                    left, 
                    mCurrentExpression);

                // Recurse
                this.ParseExpression_Power_Prime();
            }
        }

        private void ParseExpression_Mult()
        {
            this.ParseExpression_Is();
            this.ParseExpression_Mult_Prime();
        }

        private void ParseExpression_Mult_Prime()
        {
            switch (mTokenizer.CurrentToken.Kind)
            {
                case BasicTokenKind.Multiply:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        ParseExpression_Is();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.Multiply, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Mult_Prime();

                    } break;

                case BasicTokenKind.DivideFloatingPoint:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        ParseExpression_Is();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.DivideFloating, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Mult_Prime();

                    } break;

                case BasicTokenKind.DivideIntegral:
                    {
                        // Get and reset the LHS
                        var left = mCurrentExpression;
                        mCurrentExpression = null;

                        // Parse the RHS
                        this.NextTokenAndMaybeEatWhitespace();
                        ParseExpression_Is();

                        // Setup the expression
                        mCurrentExpression = new BinaryOperation(
                            MakeBinaryExpressionSpan(left, mCurrentExpression), 
                            BinaryOperationKind.DivideIntegral, 
                            left, 
                            mCurrentExpression);

                        // Recurse
                        this.ParseExpression_Mult_Prime();

                    } break;
            }
        }

        private void ParseExpression_Is()
        {
            this.ParseExpression_Term();
            this.ParseExpression_Is_Prime();
        }

        private void ParseExpression_Is_Prime()
        {
            if (this.MaybeSkipToken(BasicTokenKind.KeywordIs))
            {
                // Get and reset the LHS
                var left = mCurrentExpression;
                mCurrentExpression = null;

                // Parse the RHS
                this.EatWhitespace();
                this.ParseExpression_Term();

                // Setup the expression
                mCurrentExpression = new BinaryOperation(
                    MakeBinaryExpressionSpan(left, mCurrentExpression), 
                    BinaryOperationKind.RefIsEqual, 
                    left, 
                    mCurrentExpression);

                // Recurse
                this.ParseExpression_Is_Prime();
            }
            else if (this.MaybeSkipToken(BasicTokenKind.KeywordIsNot))
            {
                // Get and reset the LHS
                var left = mCurrentExpression;
                mCurrentExpression = null;

                // Parse the RHS
                this.EatWhitespace();
                this.ParseExpression_Term();

                // Setup the expression
                mCurrentExpression = new BinaryOperation(
                    MakeBinaryExpressionSpan(left, mCurrentExpression), 
                    BinaryOperationKind.RefIsNotEqual, 
                    left, 
                    mCurrentExpression);

                // Recurse
                this.ParseExpression_Is_Prime();
            }
        }

        private void ParseExpression_Term()
        {
            switch (mTokenizer.CurrentToken.Kind)
            {
                case BasicTokenKind.KeywordNot:
                    var notToken = mTokenizer.CurrentToken;

                    // Parse the inner content of the Not operation
                    this.NextTokenAndMaybeEatWhitespace();
                    var inner = this.ParseExpression();

                    // Work out debugging information
                    var span = notToken.Span;

                    if (inner != null)
                    {
                        span = new SourceSpan(span.Start, inner.Span.End);
                    }

                    // Wrap it in a not expression
                    mCurrentExpression = new NotExpression(span, inner);

                    // Already moved to the next token
                    break;

                case BasicTokenKind.KeywordNothing:
                    mCurrentExpression = new NullExpression(mTokenizer.CurrentToken.Span);
                    this.NextTokenAndMaybeEatWhitespace();
                    break;

                case BasicTokenKind.LiteralBoolean:
                    mCurrentExpression = new ConstantExpression(mTokenizer.CurrentToken.Span, mTokenizer.CurrentToken.ValueBoolean);
                    this.NextTokenAndMaybeEatWhitespace();
                    break;

                case BasicTokenKind.LiteralFloatingPointNumber:
                    mCurrentExpression = new ConstantExpression(mTokenizer.CurrentToken.Span, mTokenizer.CurrentToken.ValueDouble);
                    this.NextTokenAndMaybeEatWhitespace();
                    break;

                case BasicTokenKind.LiteralInteger:
                    mCurrentExpression = new ConstantExpression(mTokenizer.CurrentToken.Span, mTokenizer.CurrentToken.ValueLong);
                    this.NextTokenAndMaybeEatWhitespace();
                    break;

                case BasicTokenKind.LiteralString:
                    mCurrentExpression = new ConstantExpression(mTokenizer.CurrentToken.Span, mTokenizer.CurrentToken.ValueString);
                    this.NextTokenAndMaybeEatWhitespace();
                    break;

                case BasicTokenKind.Identifier:
                    var identifier = mTokenizer.CurrentToken;
                    this.NextTokenAndMaybeEatWhitespace();

                    if (this.MaybeSkipToken(BasicTokenKind.ParanthesisOpen))
                    {

                        // With arguments
                        var args = this.ParseMethodArgumentsWithParan();

                        // The token currently in the tokenizer is the closing paranthesis
                        // (assuming the code is correct anyway)
                        var callExpressionSpan = SourceSpan.Invalid;

                        if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
                        {
                            callExpressionSpan = new SourceSpan(identifier.Span.Start, mTokenizer.CurrentToken.Span.End);
                            this.NextToken();
                        }

                        // Add the call statement
                        mCurrentExpression = new CallExpression(callExpressionSpan, identifier.ValueString, args);

                        // Eat trailing whitespace
                        this.MaybeEatWhitespace();

                    }
                    else if (this.MaybeSkipToken(BasicTokenKind.Dot))
                    {
                        mCurrentExpression = this.ParseInvoke(identifier, FunctionCallBehavior.ParanthesisRequired);
                        this.MaybeEatWhitespace();
                    }
                    else
                    {
                        mCurrentExpression = new NamedReferenceExpression(identifier.Span, identifier.ValueString);
                    }

                    break;

                default:
                    return;
            }
        }

        private InvokeExpression ParseInvoke(BasicToken firstToken, FunctionCallBehavior behavior)
        {
            ExpressionBase current = new NamedReferenceExpression(firstToken.Span, firstToken.ValueString);

            for (; ; )
            {

                // Get the first member to invoke
                this.ExpectToken(BasicTokenKind.Identifier);
                var memberToken = mTokenizer.CurrentToken;

                this.NextToken();

                // Is it a function call?
                if (this.MaybeSkipToken(BasicTokenKind.ParanthesisOpen))
                {

                    // With arguments
                    var args = this.ParseMethodArgumentsWithParan();

                    // The token currently in the tokenizer is the closing paranthesis
                    // (assuming the code is correct anyway)
                    var invokeExpressionSpan = SourceSpan.Invalid;

                    if (this.MaybeExpectToken(BasicTokenKind.ParanthesisClose))
                    {
                        invokeExpressionSpan = new SourceSpan(memberToken.Span.Start, mTokenizer.CurrentToken.Span.End);
                        this.NextToken();
                    }

                    // Parse as a function call with brackets required
                    current = new InvokeExpression(
                        invokeExpressionSpan,
                        current,
                        memberToken.ValueString,
                        args);

                    // Consume the final paranthesis
                    this.NextToken();
                }
                else if (behavior == FunctionCallBehavior.ParanthesisOptional && mTokenizer.CurrentToken.Kind == BasicTokenKind.Whitespace)
                {
                    this.EatWhitespace();

                    // With arguments
                    var args = this.ParseMethodArgumentsNoParan();

                    // The last token in the args is what we should use as the ending
                    // location
                    var invokeExpressionSpan = args.Count > 0 ?
                        new SourceSpan(memberToken.Span.Start, args[args.Count - 1].Span.End) :
                        new SourceSpan(memberToken.Span.Start, memberToken.Span.Start);

                    // Parse as a function call - no-brackets
                    current = new InvokeExpression(
                        invokeExpressionSpan,
                        current,
                        memberToken.ValueString,
                        args);
                }
                else
                {
                    current = new InvokeExpression(
                        memberToken.Span,
                        current,
                        memberToken.ValueString,
                        null);
                }

                // Have we got more?
                if (this.MaybeSkipToken(BasicTokenKind.Dot))
                {
                    continue;
                }
                else
                {
                    return (InvokeExpression)current;
                }
            }
        }

        #endregion

        #endregion

        #region Tokenizer Helpers

        private BasicTokenKind NextToken()
        {
            var kind = mTokenizer.ReadBasicNext();

            if (mTokenSink != null)
            {
                mTokenSink.ReceiveToken(mTokenizer.CurrentToken);
            }

            return kind;
        }

        private void NextTokenAndMaybeEatWhitespace()
        {
            this.NextToken();
            this.MaybeEatWhitespace();
        }

        private void ExpectToken(BasicTokenKind kind)
        {
            if (!MaybeExpectToken(kind))
            {
                this.OnUnexpectedToken(kind, mTokenizer.CurrentToken, true);
            }
        }

        private bool MaybeExpectToken(BasicTokenKind kind)
        {
            return mTokenizer.CurrentToken.Kind == kind;
        }

        private void SkipToken(BasicTokenKind kind)
        {
            if (!MaybeSkipToken(kind))
            {
                this.OnUnexpectedToken(kind, mTokenizer.CurrentToken, true);
            }
        }

        private bool MaybeSkipToken(BasicTokenKind kind)
        {
            if (mTokenizer.CurrentToken.Kind == kind)
            {
                this.NextToken();
                return true;
            }
            else
            {
                return false;
            }
        }

        private void EatWhitespace()
        {
            this.Eat(BasicTokenKind.Whitespace);
        }

        private void EatComment()
        {
            this.Eat(BasicTokenKind.Comment);
        }

        private void EatNewLines()
        {
            this.Eat(BasicTokenKind.NewLines);
        }

        private bool MaybeEatWhitespace()
        {
            return MaybeEat(BasicTokenKind.Whitespace);
        }

        private bool MaybeEatNewLines()
        {
            return MaybeEat(BasicTokenKind.NewLines);
        }

        private bool MaybeEatWhitespaceOrNewLines()
        {
            return MaybeEat(BasicTokenKind.Whitespace, BasicTokenKind.NewLines);
        }

        private bool MaybeEatWhitespaceOrComment()
        {
            return MaybeEat(BasicTokenKind.Whitespace, BasicTokenKind.Comment);
        }

        private bool MaybeEatWhitespaceNewLinesOrComment()
        {
            return MaybeEat(BasicTokenKind.Whitespace, BasicTokenKind.NewLines, BasicTokenKind.Comment);
        }

        private void Eat(BasicTokenKind kind)
        {
            if (!MaybeEat(kind))
            {
                this.OnUnexpectedToken(kind, mTokenizer.CurrentToken, true);
            }
        }

        private bool MaybeEat(BasicTokenKind kind)
        {
            if (mTokenizer.CurrentToken.Kind == kind)
            {
                while (this.NextToken() == kind) ;
                return true;
            }
            else
            {
                return false;
            }
        }

        private bool MaybeEat(BasicTokenKind kind1, BasicTokenKind kind2)
        {
            BasicTokenKind next = mTokenizer.CurrentToken.Kind;

            if (next == kind1 || next == kind2)
            {
                do
                {
                    next = this.NextToken();
                }
                while (next == kind1 || next == kind2);

                return true;
            }
            else
            {
                return false;
            }
        }

        private bool MaybeEat(BasicTokenKind kind1, BasicTokenKind kind2, BasicTokenKind kind3)
        {
            BasicTokenKind next = mTokenizer.CurrentToken.Kind;

            if (next == kind1 || next == kind2 || next == kind3)
            {
                do
                {
                    next = this.NextToken();
                }
                while (next == kind1 || next == kind2 || next == kind3);

                return true;
            }
            else
            {
                return false;
            }
        }

        #endregion

        #region Helper Methods

        private static SourceSpan MakeBinaryExpressionSpan(ExpressionBase left, ExpressionBase right)
        {
            if (left != null && left.Span != SourceSpan.Invalid)
            {
                if (right != null && right.Span != SourceSpan.Invalid)
                {
                    return new SourceSpan(left.Span.Start, right.Span.End);
                }
                else
                {
                    return left.Span;
                }
            }
            else if (right != null && right.Span != SourceSpan.Invalid)
            {
                return right.Span;
            }

            return SourceSpan.Invalid;
        }

        private void OnUnexpectedToken(BasicTokenKind expected, BasicToken actual, bool next)
        {
            mSink.Add(
                mSource,
                string.Format("Unexpected token, expected '{0}', got '{1}' at L{2}C{3}", expected, actual.Kind, actual.Span.Start.Line, actual.Span.Start.Column),
                actual.Span,
                0,
                Severity.Error
                );

            if (next) this.NextToken();
        }

        private void OnUnexpectedToken(BasicToken actual, bool next)
        {
            mSink.Add(
                mSource,
                string.Format("Unexpected token '{0}' at L{1}C{2}", actual.Kind, actual.Span.Start.Line, actual.Span.Start.Column),
                actual.Span,
                0,
                Severity.Error
                );

            if (next) this.NextToken();
        }

        private void OnUnexpectedIdentifier(string expected, BasicToken actual, bool next)
        {
            mSink.Add(
                mSource,
                string.Format("Unexpected identifier '{1}' at L{2}C{3}, expected '{0}'", expected, actual.ValueString, actual.Span.Start.Line, actual.Span.Start.Column),
                actual.Span,
                0,
                Severity.Error
                );

            if (next) this.NextToken();
        }

        #endregion

    }
}
